package cn.itcast.webmagicDemo.task;

import cn.itcast.webmagic.JobProcesser;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

/**
 *  定义定时爬虫任务
 */
@Component // 将该类交给Spring来进行管理
@SuppressWarnings("all")
public class JobProcessor implements PageProcessor {

    // 设置爬虫页面的起始地址
    private final String INIT_URL = "https://search.51job.com/list/000000,000000,0000,00,9,99,java,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";

    // 设置请求相关的参数的对象
    private final Site site = new Site().me()
            .setCharset("utf8")
            .setTimeOut(10000)
            .setSleepTime(1000).setRetryTimes(1000).setRetrySleepTime(1000);

    @Override
    public void process(Page page) {

    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 定时任务，此时只需要添加注解@Scheduled,不需要手中写main方法实现启动
     * 此外也可以设置一些启动参数
     */
    @Scheduled(initialDelay = 1000,fixedDelay = 10000)
    public void process() {
        Spider.create(new JobProcesser())
        // 添加请求地址
        .addUrl(INIT_URL)
        // 设置线程数量
        .thread(10)
        // 设置url过滤器：采用的是布隆过滤器
        .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
        .run(); // 执行
    }
}
